FEXPSolver.h

Go to the documentation of this file.

// © FEXP, FEXPEnterprise Solver, Ing. Vaclav Rek
// Manager of a numerical simulation process.
// Compiler must support C++ ver.14 and later
#ifndef _CFEXPSOLVER_H_
#define _CFEXPSOLVER_H_
#include "FEXPCommon.h"
#include "FEXPElement.h"
#include "FEXPConcurency.h"
#include "FEXPTopologieKDTree.h"
#include "FEXPModelBuilder.h"
#include "FEXPCalculation.h"
#include "FEXPNetworkInterface.h"


class ICFEXPSolverBase
{
public:
  ICFEXPSolverBase(Ptr<ICFEXPModelBuilderBase> builder, Ptr<ICFEXPResultExport> exporter)
    : _builder(builder), _exporter(exporter) { }
  virtual ~ICFEXPSolverBase() { }

  // pure virtual member functions
  virtual void Start() = 0;
  virtual void End() = 0;
  // adding and removing model
  virtual void AddModel(const t_ModelData & data) = 0;
  virtual void RmvModel(const std::string & key ) = 0;
  // current time step dt
  virtual t_fexpcommon_ct GetCurrentDt()                   = 0;
  virtual void            SetCurrentDt(t_fexpcommon_ct dt) = 0;
  // serialized data
  virtual Ptr<t_SerializedData> GetSerializedData()                           = 0;
  virtual void                  SetSerializedData(Ptr<t_SerializedData> data) = 0;
protected:
  Ptr<t_KD_def_tree<ICFEXPElementNodeBase>> _kd_tree_map;
  Ptr<ICFEXPModelBuilderBase>               _builder;
  Ptr<ICFEXPResultExport>                   _exporter;

  void update_topology_map_for_contact();
  Ptr<std::vector<Ptr<ICFEXPElementNodeBase>>>
    check_node(const std::vector<t_fexpcommon_ct> & min_bound, const std::vector<t_fexpcommon_ct> & max_bound);
private:
  // [no private members  ] -----------------------------
};

// primary template
template<typename TValue, typename Enable = void>
class CFEXPSequentialSolver { };
template<typename TExpCalc>
class CFEXPSequentialSolver
  <TExpCalc, typename std::enable_if<std::is_base_of<ICFEXPExplicitCalcBase, TExpCalc>::value>::type>
  : public ICFEXPSolverBase
{
public:
  CFEXPSequentialSolver(Ptr<ICFEXPModelBuilderBase> builder, Ptr<ICFEXPResultExport> exporter);
  virtual ~CFEXPSequentialSolver() { }

  // override base virtual functions
  virtual void Start() override;
  virtual void End  () override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  // adding and removing model
  virtual void AddModel(const t_ModelData & data) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  virtual void RmvModel(const std::string & key ) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  // current time step dt
  virtual t_fexpcommon_ct GetCurrentDt()                   override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  virtual void            SetCurrentDt(t_fexpcommon_ct dt) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  // serialized data
  virtual Ptr<t_SerializedData> GetSerializedData()                           override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  virtual void                  SetSerializedData(Ptr<t_SerializedData> data) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
protected:
  // [no protected members] -----------------------------
private:
  Ptr<TExpCalc> _calculator;
};

//--------------------------------------------------------------------------------------
template<typename TExpCalc>
CFEXPSequentialSolver<TExpCalc, typename std::enable_if<std::is_base_of<ICFEXPExplicitCalcBase, TExpCalc>::value>::type>
::CFEXPSequentialSolver(Ptr<ICFEXPModelBuilderBase> builder, Ptr<ICFEXPResultExport> exporter)
  : ICFEXPSolverBase(builder, exporter),
  _calculator(CFEXPDataManager<TExpCalc>::SafeAllocInstance(builder->GetModelContainer(), _exporter,
    [this] { update_topology_map_for_contact(); }, [this](auto in1, auto in2) { return check_node(in1, in2); })) { }
//--------------------------------------------------------------------------------------

//--------------------------------------------------------------------------------------
template<typename TExpCalc>
void CFEXPSequentialSolver<TExpCalc, typename std::enable_if<std::is_base_of<ICFEXPExplicitCalcBase, TExpCalc>::value>::type>
::Start()
//--------------------------------------------------------------------------------------
{
  try
  {
    auto result_saved = false;
    // Check simulation end
    while (true)
    {
      // 1. Step: Check end of calculation
      if (_calculator->CheckEnd(FEXPCOMMON_DEFAULT_VALUE))
      {
        if (!result_saved)
          _calculator->TrySaveResults(FEXPCOMMON_DEFAULT_VALUE, true);
        break;
      }
      // 2.  Step: Time increment consistency
      _calculator->SimulationTimeIncrement      (FEXPCOMMON_DEFAULT_VALUE);
      // 3.  Step: Prepare data for new time level
      _calculator->PrepareDataForNewTimeLevel   (FEXPCOMMON_DEFAULT_VALUE);
      // 4.  Step: Update mapping of fe nodes for contact searching
      _calculator->UpdateModelMapping           (FEXPCOMMON_DEFAULT_VALUE);
      // 5.  Step: Transform position and velocities to local coordinate system
      _calculator->GlobalToLocalTransformation  (FEXPCOMMON_DEFAULT_VALUE);
      // 6.  Step: Integration of internal, external and contact forces
      _calculator->CalculateForces              (FEXPCOMMON_DEFAULT_VALUE);
      // 7.  Step: Calculate new displacemnets based on explicit integr. of EOM
      _calculator->CalculateNewGeometry         (FEXPCOMMON_DEFAULT_VALUE);
      // 8.  Step: Save calculation results
      result_saved = _calculator->TrySaveResults(FEXPCOMMON_DEFAULT_VALUE);
      // 9.  Step: Print out results
      _calculator->PrintOutResults              (FEXPCOMMON_DEFAULT_VALUE);
      // clear all results --> data space optimization
      _calculator->ClearResults                 (FEXPCOMMON_DEFAULT_VALUE);
      // 10. Step: Control of numerical stability
      _calculator->StabilityControl             (FEXPCOMMON_DEFAULT_VALUE);
      // 11. Step: Update time step dt
      _calculator->SetNewTimeStep(_calculator->GetCalculatedCriticTimeStep());
    }
  }
  catch (const std::exception & ex)
  {
    CFEXPLog::WriteLine();
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Error: Computation ended before the simulation end.");
    // print out the problem of exception
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "An exception occurred:");
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> " + ex.what());
  }
}

#define START_THREAD_ID 1

class ICFEXPThreadDataBase
{
public:
  ICFEXPThreadDataBase(size_t threads);
  virtual ~ICFEXPThreadDataBase() { }

  t_fexpcommon_ct GetTimeStep(size_t thread_id)                        { return _dt_map[thread_id];  }
  void            SetTimeStep(size_t thread_id, t_fexpcommon_ct value) { _dt_map[thread_id] = value; }
  t_fexpcommon_ct GetMinTStep();
  void            SetMinTStep(t_fexpcommon_ct value);

  bool GetIsEnd(size_t thread_id)             { return _is_end_map[thread_id];  }
  void SetIsEnd(size_t thread_id, bool value) { _is_end_map[thread_id] = value; }
  bool CheckEnd();
  void SetCalcTimeIncrement(size_t thread_id, size_t value) { _time_consistncy_map[thread_id] = value; }
  void CheckCalcTimeConsistency();

  Ptr<std::vector<size_t>> GetThreadId();
protected:
  std::map<size_t, t_fexpcommon_ct> _dt_map;
  std::map<size_t, bool>            _is_end_map;
  std::map<size_t, size_t>          _time_consistncy_map;
private:
  size_t _threads_number;
};


class CFEXPSolverThreadInputData
  : public ICFEXPThreadDataBase
{
public:
  CFEXPSolverThreadInputData(size_t threads) : ICFEXPThreadDataBase(threads) { }
  virtual ~CFEXPSolverThreadInputData() { }
protected:
  // [no protected members] -----------------------------
private:
  // [no private members  ] -----------------------------
};

// Cpp thread implementation class for calculation
#define PARRALEL_SOLVER_THREAD_TEMPLATE_DEF(cls_thread, cls_data) cls_thread<void(size_t, Ptr<cls_data>), size_t, Ptr<cls_data>>
// Simple parallel solver for a single machine
#define SIMPLE_PARALLEL_SOLVER CFEXPParallelSolver<CFEXPCalculation, CFEXPSolverThreadInputData, PARRALEL_SOLVER_THREAD_TEMPLATE_DEF(CFEXPCppThread, CFEXPSolverThreadInputData), CFEXPThreadGeneralBarrier>
// primary template
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier, typename Enable = void>
class CFEXPParallelSolver { };
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
class CFEXPParallelSolver
  // specialization
  <TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  : public ICFEXPSolverBase
{
public:
  CFEXPParallelSolver(Ptr<ICFEXPModelBuilderBase> builder, Ptr<ICFEXPResultExport> exporter);

  // override base virtual functions
  virtual void Start() override;
  virtual void End  () override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  // adding and removing model
  virtual void AddModel(const t_ModelData & data) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  virtual void RmvModel(const std::string &  key) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  // current time step dt
  virtual t_fexpcommon_ct GetCurrentDt()                   override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  virtual void            SetCurrentDt(t_fexpcommon_ct dt) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  // serialized data
  virtual Ptr<t_SerializedData> GetSerializedData()                           override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
  virtual void                  SetSerializedData(Ptr<t_SerializedData> data) override { FEXPCOMMON_NOT_IMPLEMENTED_EX; }
protected:
  // barrier synchronization
  enum ESynchronization { eSync1, eSync2, eSync3, eSync4, eSync5, eSync6, eSync7, eSync8, eSync9, eSync10, eSync11, eSyncCount };
  void Synchronize(ESynchronization barrier);

  // protected constructor for hybrid solver
  using t_ThrdInstGetter = std::function<Ptr<TThread>(size_t, Ptr<TThreadData>, bool)>;
  CFEXPParallelSolver(Ptr<ICFEXPModelBuilderBase> builder, Ptr<ICFEXPResultExport> exporter,
    size_t no_calc_threads, ESynchronization last_barrier, t_ThrdInstGetter thrd_inst_gttr);
  // creates instance of thread
  Ptr<TThread> get_thread_instance(size_t id, Ptr<TThreadData> data, bool is_calc_thread);
  // main calculation function
  void thread_Calculation_function(size_t thread_id, Ptr<TThreadData> data);

  size_t                          _threads;
  Ptr<TThreadData>                _thread_data;
  std::map<size_t, Ptr<TThread>>  _thread_map;
  std::map<size_t, Ptr<TExpCalc>> _calculator_map;
  inline Ptr<TExpCalc> get_calculator(size_t thread_id);

  // update structe mapping for contact algorithm 
  void update_model_mapping          (size_t thread_id);
  // update time step
  void update_time_step              (Ptr<TThreadData> data);
  // check of time synchronization correctness
  void check_time_synchronization    (Ptr<TThreadData> data);
  // save current calculation results
  std::atomic_bool _result_saved;
  void try_export_calculation_results(Ptr<TThreadData> data, size_t thread_id);
  // calculation end checking action
  bool check_calculation_end(Ptr<TThreadData> data, size_t thread_id);
private:
  // update structe mapping for contact algorithm --> synchronization objects
  std::mutex      _mtx_stuct_map_update;
  volatile size_t _thread_counter_stuct_map_update;
  // update time step --> synchronization objects
  std::mutex      _mtx_dt_update;
  volatile size_t _thread_counter_dt_update;
  // check of time synchronization correctness --> synchronization objects
  std::mutex      _mtx_time_synchr;
  volatile size_t _thread_counter_time_synchr;
  // export calculation results --> synchronization objects
  std::mutex      _mtx_save_res_synchr;
  volatile size_t _thread_counter_save_res_synchr;
  // check of calculation end --> synchronization objects
  std::mutex      _mtx_calc_end_synchr;
  volatile size_t _thread_counter_calc_end_synchr;
  // initialize
  void initialize(size_t threads, size_t no_calc_threads, ESynchronization last_barrier, t_ThrdInstGetter thrd_inst_gttr);
  // synchronization barriers
  Ptr<Tbarrier> _barrier[ESynchronization::eSyncCount];
};

// member functions
//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::CFEXPParallelSolver(Ptr<ICFEXPModelBuilderBase> builder, Ptr<ICFEXPResultExport> exporter)
  : ICFEXPSolverBase(builder, exporter), _result_saved(false)
//--------------------------------------------------------------------------------------
{
  auto threads = FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPBaseSetting,
    _builder->GetModelContainer()->GetModelElement(ICFEXPSetting::ESettingType::eModelBase, ESystemElementType::eSetting))->GetThreadNumber();
  // initialize solver
  initialize(threads, FEXPCOMMON_DEFAULT_VALUE, ESynchronization::eSync9, [this](auto val1, auto val2, auto val3) { return get_thread_instance(val1, val2, val3); });
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::CFEXPParallelSolver(Ptr<ICFEXPModelBuilderBase> builder, Ptr<ICFEXPResultExport> exporter,
    size_t no_calc_threads, ESynchronization last_barrier, t_ThrdInstGetter thrd_inst_gttr)
  : ICFEXPSolverBase(builder, exporter), _result_saved(false)
//--------------------------------------------------------------------------------------
{
#define PARALLEL_SOLVER_ERROR_THREADS "Error: Try exclude too much threads \""
  auto setting = FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPBaseSetting,
    _builder->GetModelContainer()->GetModelElement(ICFEXPSetting::ESettingType::eModelBase, ESystemElementType::eSetting));
  auto threads_setting_num = setting->GetThreadNumber();
  if (threads_setting_num <= no_calc_threads)
  {
    std::string error = PARALLEL_SOLVER_ERROR_THREADS + CFEXPBaseConvers::NumberToString<size_t>(no_calc_threads) + "\"!!!\n";
    error += "--> Allowable number of threads is \"" + CFEXPBaseConvers::NumberToString<size_t>(threads_setting_num) + "\".";
    FEXPCOMMON_EXCEPTION(error);
  }
  // set new number of calculation threads
  setting->SetThreadNumber(threads_setting_num - no_calc_threads);
  // update model mapping to threads
  _builder->ScheduleThreads(ESystemElementType::eNode);
  _builder->ScheduleThreads(ESystemElementType::eElement);
  // initialize solver
  initialize(setting->GetThreadNumber(), no_calc_threads, last_barrier, thrd_inst_gttr);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::initialize(size_t threads, size_t no_calc_threads, ESynchronization last_barrier, t_ThrdInstGetter thrd_inst_gttr)
//--------------------------------------------------------------------------------------
{
  _threads                         = threads;
  _thread_counter_stuct_map_update = FEXPCOMMON_DEFAULT_VALUE;
  _thread_counter_dt_update        = FEXPCOMMON_DEFAULT_VALUE;
  _thread_counter_time_synchr      = FEXPCOMMON_DEFAULT_VALUE;
  _thread_counter_save_res_synchr  = FEXPCOMMON_DEFAULT_VALUE;
  _thread_counter_calc_end_synchr  = FEXPCOMMON_DEFAULT_VALUE;
  _thread_data                     = CFEXPDataManager<TThreadData>::SafeAllocInstance(_threads);
  // calculation threads
  auto thread_id = *_thread_data->GetThreadId().get(); auto calc_thread_num = thread_id.size();
  FEXPCOMMON_FOREACH_ITER_FNC(thread_id,
    {
      // thread ID
      auto id = IT;
      // create thread instances
      _thread_map.insert(MAP_PAIR(id, thrd_inst_gttr(id, _thread_data, true)));
      // create calculation instance
      _calculator_map.insert(MAP_PAIR(id, CFEXPDataManager<TExpCalc>::SafeAllocInstance(_builder->GetModelContainer(), _exporter,
        [this] { update_topology_map_for_contact(); }, [this](auto in1, auto in2) { return check_node(in1, in2); })));
    });
  // no calculation threads --> other purpose
  auto thread_data_no_calc = *CFEXPDataManager<TThreadData>::SafeAllocInstance(no_calc_threads)->GetThreadId().get();
  FEXPCOMMON_FOREACH_ITER_FNC(thread_data_no_calc,
    {
      auto thread_id_gen = size_t(FEXPCOMMON_DEFAULT_VALUE);
      while (true)
      {
        if (_thread_map.count(++thread_id_gen))
          continue;
        _thread_map.insert(MAP_PAIR(thread_id_gen, thrd_inst_gttr(thread_id_gen, _thread_data, false)));
        break;
      }
    });
  // create thread barriers
  size_t idx;
  FEXPCOMMON_FOREACH(size_t(ESynchronization::eSync1), size_t(last_barrier), idx)
    _barrier[idx] = CFEXPDataManager<Tbarrier>::SafeAllocInstance(thread_id.size() + thread_data_no_calc.size());
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::Start()
//--------------------------------------------------------------------------------------
{
  FEXPCOMMON_FOREACH_ITER_FNC(_thread_map, { _thread_map[IT.first]->StartThread(); });
  FEXPCOMMON_FOREACH_ITER_FNC(_thread_map, { _thread_map[IT.first]->Join       (); });
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
Ptr<TThread> CFEXPParallelSolver<TExpCalc  , TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::get_thread_instance(size_t id, Ptr<TThreadData> data, bool is_calc_thread)
//--------------------------------------------------------------------------------------
{
  return CFEXPDataManager<TThread>::SafeAllocInstance(
    [this](auto it1, auto it2) { CFEXPParallelSolver::thread_Calculation_function(it1, it2); }, id, data);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc          , TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::thread_Calculation_function(size_t thread_id, Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  try
  {
    bool calculation_end = false; auto calculator = get_calculator(thread_id);
    while (true)
    {
      // 1.  Step: Set time consistency
      data->SetCalcTimeIncrement(thread_id, calculator->SimulationTimeIncrement(thread_id));
      Synchronize(ESynchronization::eSync1);
      // 2.  Step: Check if time is the same for all threads
      // this require only one thread to do this action
      check_time_synchronization(data);
      // check end of calculation --> wait for thread saving results
      calculation_end = check_calculation_end(data, thread_id);
      Synchronize(ESynchronization::eSync2);
      // 3.  Step: Check end of computation
      if (calculation_end)
        break;
      Synchronize(ESynchronization::eSync3);
      // 4.   Step: Prepare data for new time level
      calculator->PrepareDataForNewTimeLevel(thread_id);
      // 5.   Step: Update mapping of fe nodes for contact search
      // this require only one thread to do this action
      update_model_mapping(thread_id);
      // 6.   Step: Transform position and velocities to local coordinate system
      calculator->GlobalToLocalTransformation(thread_id);
      Synchronize(ESynchronization::eSync4);
      // 7.   Step: Integration of internal, external and contact forces
      calculator->CalculateForces(thread_id);
      Synchronize(ESynchronization::eSync5);
      // 8.   Step: Compute new displacements based on explicit integration of EM
      calculator->CalculateNewGeometry(thread_id);
      Synchronize(ESynchronization::eSync6);
      // 9.1. Step: Export results
      try_export_calculation_results(data, thread_id);
      // 9.2. Step: Print out results
      calculator->PrintOutResults(thread_id);
      Synchronize(ESynchronization::eSync7);
      // 10.  Step: Control of numerical stability
      calculator->StabilityControl(thread_id);
      data->SetTimeStep(thread_id, calculator->GetCalculatedCriticTimeStep());
      Synchronize(ESynchronization::eSync8);
      // 11.  Step: Update time step dt
      // this require only one thread to do this action
      update_time_step(data);
      // 12.  Step: Synchronize befor new loop
      Synchronize(ESynchronization::eSync9);
    }
  }
  catch (const std::exception & ex)
  {
    CFEXPLog::WriteLine();
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Error: Calculation ended before simulation end.");
    // print problem of exception
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "An exception occurred:");
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> " + ex.what());
  }
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
Ptr<TExpCalc> CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::get_calculator(size_t thread_id)
//--------------------------------------------------------------------------------------
{
  return _calculator_map[thread_id];
}

// value of thread counter of in critical section to update
#define THREAD_COUNT_UPDATER 1
//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::update_model_mapping(size_t thread_id)
//--------------------------------------------------------------------------------------
{
  // critical section for model mapping update
  std::unique_lock<std::mutex> lock(_mtx_stuct_map_update, std::defer_lock);
  lock.lock();
  ++_thread_counter_stuct_map_update; // increment thread counter
  if (_thread_counter_stuct_map_update == THREAD_COUNT_UPDATER)
    get_calculator(thread_id)->UpdateModelMapping(thread_id);
  if (_thread_counter_stuct_map_update == _threads)
    _thread_counter_stuct_map_update = FEXPCOMMON_DEFAULT_VALUE;
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::update_time_step(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // critical section for setting of new time step
  std::unique_lock<std::mutex> lock(_mtx_dt_update, std::defer_lock);
  lock.lock();
  ++_thread_counter_dt_update; // increment thread counter
  if (_thread_counter_dt_update == THREAD_COUNT_UPDATER)
  {
    auto min_dt = data->GetMinTStep();
    FEXPCOMMON_FOREACH_ITER_FNC(_thread_map,
      {
        if (_calculator_map.count(IT.first))
          _calculator_map[IT.first]->SetNewTimeStep(min_dt);
      });
  }
  if (_thread_counter_dt_update == _threads)
    _thread_counter_dt_update = FEXPCOMMON_DEFAULT_VALUE;
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::check_time_synchronization(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // critical section for setting of new time step
  std::unique_lock<std::mutex> lock(_mtx_time_synchr, std::defer_lock);
  lock.lock();
  ++_thread_counter_time_synchr; // increment thread counter
  if (_thread_counter_time_synchr == THREAD_COUNT_UPDATER)
    // if values for all threads are not the same -- exception is thrown
    data->CheckCalcTimeConsistency();
  if (_thread_counter_time_synchr == _threads)
    _thread_counter_time_synchr = FEXPCOMMON_DEFAULT_VALUE;
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::try_export_calculation_results(Ptr<TThreadData> data, size_t thread_id)
//--------------------------------------------------------------------------------------
{
  // critical section for export of results from numerical simulation
  std::unique_lock<std::mutex> lock(_mtx_save_res_synchr, std::defer_lock);
  lock.lock();
  ++_thread_counter_save_res_synchr; // increment thread counter
  if (_thread_counter_save_res_synchr == THREAD_COUNT_UPDATER)
  {
    auto calculator = get_calculator(thread_id);
    _result_saved.store(calculator->TrySaveResults(FEXPCOMMON_DEFAULT_VALUE));
    // clear all results --> data space optimization
    calculator->ClearResults(FEXPCOMMON_DEFAULT_VALUE);
  }
  if (_thread_counter_save_res_synchr == _threads)
    _thread_counter_save_res_synchr = FEXPCOMMON_DEFAULT_VALUE;
  lock.unlock();
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
bool CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::check_calculation_end(Ptr<TThreadData> data, size_t thread_id)
//--------------------------------------------------------------------------------------
{
  auto calculator = get_calculator(thread_id);
  // check calulation end
  auto result = calculator->CheckEnd(thread_id);
  data->SetIsEnd(thread_id, result);
  // critical section for calculation results export
  std::unique_lock<std::mutex> lock(_mtx_calc_end_synchr, std::defer_lock);
  lock.lock();
  ++_thread_counter_calc_end_synchr; // increment thread counter
  if (_thread_counter_calc_end_synchr == THREAD_COUNT_UPDATER)
  {
    // if is end
    if (result && !_result_saved.load())
    {
      _result_saved.store(calculator->TrySaveResults(FEXPCOMMON_DEFAULT_VALUE, true));
      // clear all results --> data space optimization
      calculator->ClearResults(FEXPCOMMON_DEFAULT_VALUE);
    }
  }
  if (_thread_counter_calc_end_synchr == _threads)
    _thread_counter_calc_end_synchr = FEXPCOMMON_DEFAULT_VALUE;
  lock.unlock();
  return result;
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier, typename std::enable_if<
  std::is_base_of<ICFEXPExplicitCalcBase   , TExpCalc   >::value &&
  std::is_base_of<ICFEXPThreadDataBase     , TThreadData>::value &&
  std::is_base_of<ICFEXPThreadBase         , TThread    >::value &&
  std::is_base_of<ICFEXPSynchrThreadBarrier, Tbarrier   >::value>::type>
  ::Synchronize(ESynchronization barrier)
//--------------------------------------------------------------------------------------
{
  _barrier[barrier]->SynchronizeThreads();
}

#define HYBRID_PARALLEL_SOLVER_EXCLUDE_THREADS 1
#define HYBRID_PARALLEL_SOLVER CFEXPHybridParallelSolver<CFEXPCalculation, CFEXPSolverThreadInputData, PARRALEL_SOLVER_THREAD_TEMPLATE_DEF(CFEXPCppThread, CFEXPSolverThreadInputData), CFEXPThreadGeneralBarrier>

template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
class CFEXPHybridParallelSolver
  : public CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
{
public:
  CFEXPHybridParallelSolver(std::function<Ptr<CFEXPFEInpContBase>(std::string &, Ptr<std::vector<std::string>>)> reader, Ptr<ICFEXPModelBuilderBase> builder,
    std::function<std::string(size_t)> model_key_gttr, ICFEXPNetClientNodeService & net_service);

  // override base virtual functions
  virtual void Start() override;
protected:
  Ptr<TThread> get_thread_instance(size_t id, Ptr<TThreadData> data, bool is_calc_thread);
  // main calculation function
  void thread_Calculation_function(size_t thread_id, Ptr<TThreadData> data);
private:
  bool _socket_closed;
  // reader of model data
  std::function<Ptr<CFEXPFEInpContBase>(std::string &, Ptr<std::vector<std::string>>)> _data_reader;
  // communication with server
  void main_loop_net_communication(size_t id, Ptr<TThreadData> data);
  // inform about calculation end
  bool end_calculation            (Ptr<TThreadData> data);
  // result export
  void result_export              (Ptr<TThreadData> data, bool forced = false);
  void try_result_export_rntm     (Ptr<TThreadData> data, bool forced = false);
  void try_result_export_full     (Ptr<TThreadData> data, bool forced = false);
  // stabil time step synchronization
  void time_step_synchronization  (Ptr<TThreadData> data);
  // model contact --> data exchange
  // send bounding box data
  void macro_model_BB_data_send   (Ptr<TThreadData> data);
  // send macro model serialized runtime calculation data
  void macro_model_calc_data_send (Ptr<TThreadData> data);
  // receive macro model input data
  void macro_model_input_data_recv(Ptr<TThreadData> data);
  // receive macro model serialized runtime calculation data
  void macro_model_calc_data_recv (Ptr<TThreadData> data);
  // checking socket error
  bool is_socket_closed(Ptr<std::map<size_t, Ptr<t_ModelData>>> data);
  std::function<std::string(size_t)> _model_key_gttr;
  ICFEXPNetClientNodeService &       _net_calc_service;
};

// member functions
//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::CFEXPHybridParallelSolver(
  std::function<Ptr<CFEXPFEInpContBase>(std::string &, Ptr<std::vector<std::string>>)> reader, Ptr<ICFEXPModelBuilderBase> builder,
  std::function<std::string(size_t)> model_key_gttr, ICFEXPNetClientNodeService & net_service)
  : CFEXPParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>(builder, Ptr<ICFEXPResultExport>(), HYBRID_PARALLEL_SOLVER_EXCLUDE_THREADS, ESynchronization::eSync11,
    [this](auto val1, auto val2, auto val3) { return get_thread_instance(val1, val2, val3); }), _model_key_gttr(model_key_gttr),
  _net_calc_service(net_service), _socket_closed(false), _data_reader(reader) { }
//--------------------------------------------------------------------------------------

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::Start()
//--------------------------------------------------------------------------------------
{
  CFEXPParallelSolver::Start();
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
Ptr<TThread> CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::get_thread_instance(size_t id, Ptr<TThreadData> data, bool is_calc_thread)
//--------------------------------------------------------------------------------------
{
  return is_calc_thread ?
    CFEXPDataManager<TThread>::SafeAllocInstance(// create calculation thread
      [this](auto it1, auto it2) { CFEXPHybridParallelSolver::thread_Calculation_function(it1, it2); }, id, data) :
    CFEXPDataManager<TThread>::SafeAllocInstance(// create thread for network communication purpose
      [this](auto it1, auto it2) { CFEXPHybridParallelSolver::main_loop_net_communication(it1, it2); }, id, data);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::thread_Calculation_function(size_t thread_id, Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  try
  {
    bool calculation_end = false; auto calculator = get_calculator(thread_id);
    while (!_socket_closed)
    {
      // 1.  Step: Set time consistency
      data->SetCalcTimeIncrement(thread_id, calculator->SimulationTimeIncrement(thread_id));
      Synchronize(ESynchronization::eSync1);
      // 2.  Step: Check if time is the same for all threads
      // this require only one thread to do this action
      check_time_synchronization(data);
      // check end of calculation
      calculation_end = check_calculation_end(data, thread_id);
      Synchronize(ESynchronization::eSync2);
      // 3.  Step: While is end wait for result sending --> process non-calc thread communicates with server
      Synchronize(ESynchronization::eSync3);
      // 4.  Step: Check end of calculation
      if (calculation_end)
        break;
      Synchronize(ESynchronization::eSync4);
      // 5.  Step: Prepare data for new time level
      calculator->PrepareDataForNewTimeLevel(thread_id);
      // 6.  Step: Update mapping of fe nodes for contact search
      // this require only one thread to do this action
      update_model_mapping(thread_id);
      // 7.  Step: Transform position and velocities to local coordinate system
      calculator->GlobalToLocalTransformation(thread_id);
      Synchronize(ESynchronization::eSync5);
      // 8.  Step: Integration of internal, external and contact forces
      calculator->CalculateForces(thread_id);
      Synchronize(ESynchronization::eSync6);
      // 9.  Step: Calculate new motion based on explicit integration equations of motion
      calculator->CalculateNewGeometry(thread_id);
      Synchronize(ESynchronization::eSync7);
      // 10. Step: Save calculation results --> process non-calc thread communicates with server
      Synchronize(ESynchronization::eSync8);
      // 11. Step: Control of calculation stability
      calculator->StabilityControl(thread_id);
      data->SetTimeStep(thread_id, calculator->GetCalculatedCriticTimeStep());
      Synchronize(ESynchronization::eSync9);
      // 13. Step: Synchronize time steps with network clients --> process non-calc thread communicates with server
      Synchronize(ESynchronization::eSync10);
      // 15. Step: Network data exchange --> process non-calc thread communicates with server
      Synchronize(ESynchronization::eSync11);
    }
  }
  catch (const std::exception & ex)
  {
    CFEXPLog::WriteLine();
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Error: Calculation ended before simulation end.");
    // print problem of exception
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "An exception occurred:");
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> " + ex.what());
  }
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::main_loop_net_communication(size_t id, Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  try
  {
    bool calculation_end = false;
    while (!_socket_closed)
    {
      Synchronize(ESynchronization::eSync1 );
      Synchronize(ESynchronization::eSync2 );
      // check calculation end --> send resuts
      calculation_end = end_calculation(data);
      Synchronize(ESynchronization::eSync3 );
      // check calculation end --> end of loop
      if (calculation_end)
        break;
      Synchronize(ESynchronization::eSync4 );
      Synchronize(ESynchronization::eSync5 );
      Synchronize(ESynchronization::eSync6 );
      Synchronize(ESynchronization::eSync7 );
      // export of results
      result_export              (data);
      Synchronize(ESynchronization::eSync8 );
      Synchronize(ESynchronization::eSync9 );
      // stability --> time step synchonization
      time_step_synchronization  (data);
      Synchronize(ESynchronization::eSync10);
      // macro contact --> data exchange
      // 1. send macro model bounding boxes
      macro_model_BB_data_send   (data);
      // 2. send macro model serialized calculation data
      macro_model_calc_data_send (data);
      // 3. receive macro model input data
      macro_model_input_data_recv(data);
      // 4. receive macro model serialized calculation data
      macro_model_calc_data_recv (data);
      Synchronize(ESynchronization::eSync11);
    }
  }
  catch (const std::exception & ex)
  {
    CFEXPLog::WriteLine();
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Error: Calculation ended before simulation end.");
    // print problem of exception
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "An exception occurred:");
    CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "--> " + ex.what());
  }
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
bool CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::end_calculation(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // check end of calculation
  auto is_end = data->CheckEnd();

  // 1. receive message request for getting results
  auto instruction = _net_calc_service.ReadInstruction();
  if (is_socket_closed(instruction))
    return is_end;
  if (!instruction->count(t_ENetMessage::eGetIsEndCalc))
    FEXPCOMMON_EXCEPTION("Error: 1. End of calculation check --> Wrong message!!!");

  // 2. send info
  auto cekey = std::string("ce");
  auto cedta = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
  cedta->insert(MAP_PAIR(cekey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
  _net_calc_service.SendInstruction(!is_end ? t_ENetMessage::eContinue : t_ENetMessage::eSetIsEndCalc, cedta);
  if (is_end)
    result_export(data, true); // send calculation results of final time step
  return is_end;
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::result_export(Ptr<TThreadData> data, bool forced/*= false*/)
//--------------------------------------------------------------------------------------
{

  // try export runtime calc data
  CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Try export runtime results.");
  try_result_export_rntm(data, forced);
  // try export full results
  CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Try export results."        );
  try_result_export_full(data, forced);
  CFEXPLog::WriteLine(FEXPCOMMON_DFLT_TXT + "Info: Export of results finished.");
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::try_result_export_rntm(Ptr<TThreadData> data, bool forced/*= false*/)
//--------------------------------------------------------------------------------------
{
  // receive message request for getting results from runtime calculation data
  auto instruction = _net_calc_service.ReadInstruction();
  if (is_socket_closed(instruction))
    return;
  if (!instruction->count(t_ENetMessage::eGetResults))
    FEXPCOMMON_EXCEPTION("Error: Export of results (runtime output) --> Wrong message!!!");

  // save calc. data to print out
  auto rsokey = std::string("rso");
  auto output = std::string(FEXPCOMMON_EMPTY_STRING);
  FEXPCOMMON_FOREACH_ITER(_calculator_map)
  {
    auto nid = CFEXPBaseConvers::StringToNumber<size_t>(instruction->at(t_ENetMessage::eGetResults)->at(rsokey)->at(0));
    auto dof = CFEXPBaseConvers::StringToNumber<size_t>(instruction->at(t_ENetMessage::eGetResults)->at(rsokey)->at(1));
    output   = ICFEXPExplicitCalcBase::GetCalcBehavOutString(FEXPCOMMON_STACAST(ICFEXPModelDataIntf, ICFEXPElementNodeBase,
      _builder->GetModelContainer()->GetModelElement(nid, ESystemElementType::eNode, IT->first)), dof);
    if (output != FEXPCOMMON_EMPTY_STRING)
      break;
  }
  auto rsodta = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
  rsodta->insert(MAP_PAIR(rsokey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
  rsodta->at(rsokey)->push_back(output);
  _net_calc_service.SendInstruction(t_ENetMessage::eSetResults, rsodta);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::try_result_export_full(Ptr<TThreadData> data, bool forced/*= false*/)
//--------------------------------------------------------------------------------------
{
  // receive message request for getting results
  auto instruction = _net_calc_service.ReadInstruction();
  if (is_socket_closed(instruction))
    return;
  if (!instruction->count(t_ENetMessage::eGetResults))
    FEXPCOMMON_EXCEPTION("Error: Export of results --> Wrong message!!!");

  auto calculator = get_calculator(_calculator_map.size());
  // send results if need
  auto rskey = std::string("rs");
  auto rsdta = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
  rsdta->insert(MAP_PAIR(rskey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
  if (!calculator->TrySaveResults(FEXPCOMMON_DEFAULT_VALUE, forced))
  {
    _net_calc_service.SendInstruction(t_ENetMessage::eSetResults, rsdta);
    return;
  }

  // get current result saved in container
  Ptr<CFEXPCalculationModelNodeResult> result;
  _builder->GetModelContainer()->IterateModElems([this, &result](auto item)
  {
    if (result)
      FEXPCOMMON_EXCEPTION("Error: Export of results --> More results remains in container!!!");
    result = FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPCalculationModelNodeResult, item);
    return true;
  }, ESystemElementType::eResult, FEXPCOMMON_DEFAULT_VALUE);
  if (!result)
  {
    _net_calc_service.SendInstruction(t_ENetMessage::eSetResults, rsdta);
    return;
  }

  // create result map to send (serizalization)
  auto block  = NmspcFileResData::EFileBlocks::eBl1;
  auto table1 = NmspcFileResData::EFileTab::eRTNDS;
  auto table2 = NmspcFileResData::EFileTab::eRTFERIAG;
  // block definition
  rsdta->at(rskey)->push_back(std::string(FEXPCOMMON_KEY_BLOCK_IDNTF) + RESULT_FILE_STRUCT_BLCS[block]);
  // node results table definition
  auto table_def_nd = std::string(FEXPCOMMON_KEY_TABLE_IDNTF) + RESULT_FILE_STRUCT_TABS[table1] + FEXPCOMMON_DELIMITER + CFEXPNodeResultData::GetDefOrder();
  rsdta->at(rskey)->push_back(table_def_nd);
  // serialize node result data
  FEXPCOMMON_FOREACH_ITER_FNC(result->GetNodeResults(), { rsdta->at(rskey)->push_back(IT->GetSerializedData()); });

  // finite element connectivity results table definition
  auto table_def_fe = std::string(FEXPCOMMON_KEY_TABLE_IDNTF) + RESULT_FILE_STRUCT_TABS[table2] + FEXPCOMMON_DELIMITER + CFEXPResultElementConnect::GetDefOrder();
  rsdta->at(rskey)->push_back(table_def_fe);
  // serialize connectivity data
  // serialize node result data
  FEXPCOMMON_FOREACH_ITER_FNC(result->GetConnectivity(), { rsdta->at(rskey)->push_back(IT->GetSerializedData()); });

  // send serialized result data data
  _net_calc_service.SendInstruction(t_ENetMessage::eSetResults, rsdta);
  // clear all results --> data space optimization
  calculator->ClearResults(FEXPCOMMON_DEFAULT_VALUE);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::time_step_synchronization(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // 1. receive message request for new time step
  auto instruction = _net_calc_service.ReadInstruction();
  if (is_socket_closed(instruction))
    return;
  if (!instruction->count(t_ENetMessage::eGetStabilDt))
    FEXPCOMMON_EXCEPTION("Error: 1. Time step --> Wrong message!!!");

  // 2. send time step and synchronize with network clients
  auto dtkey = std::string("dt");
  auto mindt = data->GetMinTStep();
  auto tdata = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
  tdata->insert(MAP_PAIR(dtkey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
  tdata->at(dtkey)->push_back(CFEXPBaseConvers::NumberToString<t_fexpcommon_ct>(mindt));
  _net_calc_service.SendInstruction(t_ENetMessage::eSetStabilDt, tdata);

  // 3. update time step after sending current one
  instruction = _net_calc_service.ReadInstruction();
  if (is_socket_closed(instruction))
    return;
  if (!instruction->count(t_ENetMessage::eSetStabilDt))
    FEXPCOMMON_EXCEPTION("Error: 2. Time step --> Wrong message!!!");
  // set time step for each calculation thread
  data->SetMinTStep(CFEXPBaseConvers::StringToNumber<t_fexpcommon_ct>(
    ICFEXPNetClientNodeService::GetReadModelData(instruction, t_ENetMessage::eSetStabilDt)[dtkey]->at(FEXPCOMMON_DEFAULT_INDX)));
  // update time step
  update_time_step(data);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::macro_model_BB_data_send(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // 1. compose bounding boxes
  //--> get bounding boxes for each macro model based on its unique ID
  auto bbox = std::map<size_t, Ptr<CFEXGeomTools::t_BoundBox>>();
  _builder->GetModelContainer()->IterateModElems([this, &bbox](auto item)
  {
    auto id = item->GetModelId();
    if (!bbox.count(id))
      bbox.insert(MAP_PAIR(id, Ptr<CFEXGeomTools::t_BoundBox>()));
    bbox[id] = CFEXGeomTools::GetBoundingBox(bbox[id],
      FEXPCOMMON_DYNCAST(ICFEXPModelDataIntf, CFEXPFECoordinates<t_fexpcommon_ct>, item));
    return true;
  }, ESystemElementType::eNode);
  FEXPCOMMON_FOREACH_ITER_FNC(bbox, { CFEXGeomTools::ExtBoundingBox(IT.second, DEFAULT_BB_RATIO); });

  // 2. send bounding box of all models
  auto bbidkey = std::string("bbid"); // bounding box id key
  auto bbdtkey = std::string("bbdt"); // bounding box data key
  FEXPCOMMON_FOREACH_ITER(bbox)
  {
    // 2.1 receive message request for sending bounding box ID
    auto instructionid = _net_calc_service.ReadInstruction();
    if (is_socket_closed(instructionid))
      return;
    if (!instructionid->count(t_ENetMessage::eGetBoundBoxId))
      FEXPCOMMON_EXCEPTION("Error: Bounding box --> Wrong message!!!");

    // 2.2 send bounding box ID
    auto bbiddata = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
    bbiddata->insert(MAP_PAIR(bbidkey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
    bbiddata->at(bbidkey)->push_back(CFEXPBaseConvers::NumberToString((*IT).first));
    _net_calc_service.SendInstruction(t_ENetMessage::eSetBoundBoxId, bbiddata);

    // 2.3 receive message request for sending bounding box
    auto instructiondt = _net_calc_service.ReadInstruction();
    if (is_socket_closed(instructiondt))
      return;
    if (!instructiondt->count(t_ENetMessage::eGetBoundBox))
      FEXPCOMMON_EXCEPTION("Error: Bounding box --> Wrong message!!!");

    // 2.4 send bounding box
    auto bbdtdata = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
    bbdtdata->insert(MAP_PAIR(bbdtkey, CFEXPDataManager<std::vector<std::string>>::SafeAllocInstance()));
    auto serialbb = CFEXGeomTools::GetSerializedBoundingBox((*IT).second);
    FEXPCOMMON_FOREACH_ITER_FNC(*serialbb.get(), { bbdtdata->at(bbdtkey)->push_back(IT); });
    _net_calc_service.SendInstruction(t_ENetMessage::eSetBoundBox, bbdtdata);
  }

  // end of data sending
  // receive message request for sending bounding box ID
  auto instructionid = _net_calc_service.ReadInstruction();
  if (is_socket_closed(instructionid))
    return;
  if (!instructionid->count(t_ENetMessage::eGetBoundBoxId))
    FEXPCOMMON_EXCEPTION("Error: Bounding box --> Wrong message!!!");
  // send empty bounding box ID to end this session
  auto bbiddata = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
  bbiddata->insert(MAP_PAIR(bbidkey, CFEXPDataManager<t_ModelDataDta>::SafeAllocInstance()));
  _net_calc_service.SendInstruction(t_ENetMessage::eSetBoundBoxId, bbiddata);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::macro_model_calc_data_send(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // 1. get request about getting of macro model serialized calculation data and send them to server
  auto trkeyget   = "trget"; // get transfer data key
  auto trkeyset   = "trset"; // set transfer data key
  auto reschedule = false;
  while (true)
  {
    // 1.1 receive message request for sending respective macro model calc. data
    auto instruction = _net_calc_service.ReadInstruction();
    if (is_socket_closed(instruction))
      break;
    if (!instruction->count(t_ENetMessage::eGetMacroModelCalcData))
      FEXPCOMMON_EXCEPTION("Error: Macro model calculation data --> Wrong message!!!");
    // macro model data ID
    auto requestdata = ICFEXPNetClientNodeService::GetReadModelData(instruction, t_ENetMessage::eGetMacroModelCalcData);
    if (requestdata[trkeyget]->empty())
    {
      _net_calc_service.SendInstruction(t_ENetMessage::eSetMacroModelCalcData, CFEXPDataManager<t_ModelData>::SafeAllocInstance());
      break;
    }

    // 1.2 Send macro model calc. data
    auto macro_id_str = requestdata[trkeyget]->at(FEXPCOMMON_DEFAULT_INDX);
    // compose serialized calculation data -- NODES
    auto macro = _builder->NetSerializeStructure(macro_id_str);
    if (!macro || macro->empty())
      FEXPCOMMON_EXCEPTION("Error: Macro model input data ID --> No such data!!!");
    auto macrocalcdata = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
    macrocalcdata->insert(MAP_PAIR(trkeyset, macro));
    _net_calc_service.SendInstruction(t_ENetMessage::eSetMacroModelCalcData, macrocalcdata);
    // removing of entire model
    _builder->RemoveStructure(macro_id_str);
    reschedule = true;
  }
  // reschedule model if need
  if (!reschedule)
    return;
  _builder->ScheduleThreads(ESystemElementType::eNode   );
  _builder->ScheduleThreads(ESystemElementType::eElement);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::macro_model_input_data_recv(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // 1. get request about sending of macro model input data
  auto trkeyset      = "trset"; // set transfer data key
  auto macro_in_data = CFEXPDataManager<t_ModelData>::SafeAllocInstance();
  while (true)
  {
    // 1.1 receive message request for sending respective macro model input data ID
    auto instruction1 = _net_calc_service.ReadInstruction();
    if (is_socket_closed(instruction1))
      break;
    if (!instruction1->count(t_ENetMessage::eSetMacroModelId))
      FEXPCOMMON_EXCEPTION("Error: Macro model input (set) data ID --> Wrong message!!!");
    // macro model input data ID
    auto requestdata1 = ICFEXPNetClientNodeService::GetReadModelData(instruction1, t_ENetMessage::eSetMacroModelId);
    if (requestdata1[trkeyset]->empty())
      break;
    auto model_id = requestdata1[trkeyset]->at(FEXPCOMMON_DEFAULT_INDX);

    // 1.2 read macro model input data
    auto instruction2 = _net_calc_service.ReadInstruction();
    if (is_socket_closed(instruction2))
      break;
    if (!instruction2->count(t_ENetMessage::eSetMacroModel))
      FEXPCOMMON_EXCEPTION("Error: Macro model input (set) data --> Wrong message!!!");
    // macro model data
    auto requestdata2 = ICFEXPNetClientNodeService::GetReadModelData(instruction2, t_ENetMessage::eSetMacroModel);
    if (requestdata2[trkeyset]->empty())
      break;
    macro_in_data->insert(MAP_PAIR(model_id, CFEXPDataManager<t_ModelDataDta>::SafeAllocInstance()));
    FEXPCOMMON_FOREACH_ITER_FNC(*requestdata2[trkeyset].get(), { macro_in_data->at(model_id)->push_back(IT); });
  }
  // create models
  if (macro_in_data->empty())
    return;
  // build new models
  FEXPCOMMON_FOREACH_ITER_FNC(*macro_in_data.get(),
    {
      auto key = IT.first;
      auto dta = IT.second;
      if (!_builder->BuildModel(key, IT.second, FEXPCOMMON_DYNCAST(CFEXPFEInpContBase, ICFEXPDataContIntf, _data_reader(key, dta))))
        FEXPCOMMON_EXCEPTION("Error: Macro model building failed!!!");
    });
  // reschedule model
  _builder->ScheduleThreads(ESystemElementType::eNode   );
  _builder->ScheduleThreads(ESystemElementType::eElement);
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
void CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::macro_model_calc_data_recv(Ptr<TThreadData> data)
//--------------------------------------------------------------------------------------
{
  // 1. get request about sending of current macro model calc. data
  auto trkeyset       = "trset"; // set transfer data key
  auto macro_clc_data = CFEXPDataManager<std::map<std::string, Ptr<t_SerializedData>>>::SafeAllocInstance();
  while (true)
  {
    // 1.1 receive message request for sending respective macro model calc. data ID
    auto instruction1 = _net_calc_service.ReadInstruction();
    if (is_socket_closed(instruction1))
      break;
    if (!instruction1->count(t_ENetMessage::eSetMacroModelCalcDataId))
      FEXPCOMMON_EXCEPTION("Error: Macro model calculation (set) data ID --> Wrong message!!!");
    // macro model data ID
    auto requestdata1 = ICFEXPNetClientNodeService::GetReadModelData(instruction1, t_ENetMessage::eSetMacroModelCalcDataId);
    if (requestdata1[trkeyset]->empty())
      break;
    auto model_id = requestdata1[trkeyset]->at(FEXPCOMMON_DEFAULT_INDX);

    // 1.2 read macro model calc. data
    auto instruction2 = _net_calc_service.ReadInstruction();
    if (is_socket_closed(instruction2))
      break;
    if (!instruction2->count(t_ENetMessage::eSetMacroModelCalcData))
      FEXPCOMMON_EXCEPTION("Error: Macro model calculation (set) data --> Wrong message!!!");
    // macro model data
    auto requestdata2 = ICFEXPNetClientNodeService::GetReadModelData(instruction2, t_ENetMessage::eSetMacroModelCalcData);
    macro_clc_data->insert(MAP_PAIR(model_id, CFEXPDataManager<t_SerializedData>::SafeAllocInstance()));
    FEXPCOMMON_FOREACH_ITER_FNC(*requestdata2[trkeyset].get(), { macro_clc_data->at(model_id)->push_back(IT); });
  }
  if (macro_clc_data->empty())
    return;
  // load serialized calc data to respective model
  FEXPCOMMON_FOREACH_ITER_FNC(*macro_clc_data.get(), { _builder->NetDeserializeStructure(IT.first, IT.second); });
}

//--------------------------------------------------------------------------------------
template<typename TExpCalc, typename TThreadData, typename TThread, typename Tbarrier>
bool CFEXPHybridParallelSolver<TExpCalc, TThreadData, TThread, Tbarrier>
  ::is_socket_closed(Ptr<std::map<size_t, Ptr<t_ModelData>>> data)
//--------------------------------------------------------------------------------------
{
  return !_socket_closed ?
    _socket_closed = data->count(t_ENetMessage::eError) != FEXPCOMMON_DEFAULT_VALUE :
    _socket_closed;
}

#endif // !_CFEXPSOLVER_H_